import requests
from bs4 import BeautifulSoup
# 导入lxml的etree模块
from lxml import etree
import json

headers={
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
}
url = 'http://www.techlabplt.com:8080/BD-PC/proxy.html'

response = requests.get(url=url, headers=headers)
response.encoding='utf-8'
html = etree.HTML(response.text)

pageHtml = BeautifulSoup(response.text, "html.parser")
# 获取html内的div，div的class为proxy-title。里面包含着此网站的标题数据
title = pageHtml.find("div", attrs={"class": "proxy-title"}).text.replace(" ", '').replace("\n", '')
# 输出标题
print(title)

# 代理网站的tr列表
dataDiv = html.xpath('/html/body/div[3]/div[2]/div/div[2]/table/tbody/tr')
dataList=[]
for data in dataDiv:
    # 得到每一行中的每一列
    oneData = data.findall('td')
    # 存储到jsonData对象中
    jsonData = {
        "ip地址": oneData[0].text,
        "端口": oneData[1].text,
        "代理类型": oneData[3].text,
        "地理位置": oneData[5].text
    }
    #把jsonData对象添加到dataList数组
    dataList.append(jsonData)

#7.新建命名为data.json的json文件，并将dataList数据存储
with open('D:/pycharm community/lab/代理ip.json', 'w', encoding='utf-8') as jsonFile:
    jsonFile.write(json.dumps(dataList, ensure_ascii=False, indent=4))
